{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_phi_state, n_phi_action = 5,3 # 状态特征数和行为特征数\n",
    "# 假设为离散行为空间[0,1,2]代表[静止,左移,右移]\n",
    "M = 64 # 总的样本数\n",
    "theta = np.random.random((n_phi_state + n_phi_action,1)) # 8*1\n",
    "actions = np.eye(3,3) # 3*3\n",
    "# actions[0] 静止\n",
    "# actions[1] 左移\n",
    "# actions[2] 右移\n",
    "states = np.random.random((M,n_phi_state))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_phi(state, action):\n",
    "    state = state.reshape(1,-1)\n",
    "    action = action.reshape(1,-1)\n",
    "    phi_sa= np.concatenate([state, action], axis = 1) # 3*8\n",
    "    return phi_sa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_scores(state):\n",
    "    score = np.zeros(n_phi_action)\n",
    "    for i in range(len(actions)):\n",
    "        score[i] = get_phi(state, actions[i]).dot(theta)\n",
    "    score -= np.mean(score)\n",
    "    return score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.13376092, -0.26889592,  0.135135  ])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_scores(states[0])\n",
    "scores = [3,5,8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
